def evaluate_model(model_function, features, labels, model_name="Model", num_folds=10, epochs=100, batch_size=128, seed=42):
"""
Evaluate a model using k-fold cross-validation and return performance metrics including confusion matrix.
"""
accuracies = []
histories = []
class_names = ["AC", "CH", "CP", "DB", "DR", "EI", "GS", "JA", "SI", "SM"]
cumulative_conf_matrix = np.zeros((10, 10))
# Cross-validation loop
for i in range(num_folds):
print(f"\nTraining {model_name} - Fold {i+1}/{num_folds}")
# Clear previous session and set random seeds
tf.keras.backend.clear_session()
tf.random.set_seed(seed)
early_stopping = tf.keras.callbacks.EarlyStopping(
monitor='val_loss',
patience=5,
restore_best_weights=True,
verbose=1
)
# Define train, validation and test sets for this iteration
test_fold = i
val_fold = (i + 1) % num_folds
train_folds = [f for f in range(num_folds) if f not in [test_fold, val_fold]]
# Combine training data (8 folds)
train_features = np.concatenate([features[f] for f in train_folds])
train_labels = np.concatenate([labels[f] for f in train_folds])
# Get validation data (1 fold)
val_features = features[val_fold]
val_labels = labels[val_fold]
# Get test data (1 fold)
test_features = features[test_fold]
test_labels = labels[test_fold]
# Create and train model
model = model_function(train_features.shape[1])
# Train using validation data
history = model.fit(
train_features, train_labels,
epochs=epochs,
batch_size=batch_size,
validation_data=(val_features, val_labels),
callbacks=[early_stopping], # Add this line
verbose=0
)
histories.append(history.history)
# Get predictions for confusion matrix
predictions = np.argmax(model.predict(test_features), axis=1)
# Compute confusion matrix for this fold
fold_conf_matrix = confusion_matrix(test_labels, predictions, labels=range(10))
cumulative_conf_matrix += fold_conf_matrix
# Calculate and store accuracy
test_loss, test_accuracy = model.evaluate(test_features, test_labels, verbose=0)
accuracies.append(test_accuracy)
print(f"Fold {i+1} Accuracy: {test_accuracy:.4f}")
# Calculate final metrics
mean_accuracy = np.mean(accuracies)
std_accuracy = np.std(accuracies)
print(f"\n{model_name} Final Results:")
print(f"Mean accuracy: {mean_accuracy:.4f}")
print(f"Std accuracy: {std_accuracy:.4f}")
# Print raw numbers matrix
print("\nRaw Counts Confusion Matrix:")
plt.figure(figsize=(12, 10))
sns.heatmap(cumulative_conf_matrix, annot=True, fmt="g", cmap="Blues",
xticklabels=class_names, yticklabels=class_names)
plt.xlabel("Predicted label")
plt.ylabel("True label")
plt.title("Cumulative Confusion Matrix (Raw Counts)")
plt.tight_layout()
plt.show()
# Create normalized confusion matrix (percentages by row)
row_sums = cumulative_conf_matrix.sum(axis=1)
normalized_conf_matrix = (cumulative_conf_matrix.T / row_sums).T * 100
# Plot normalized confusion matrix
plt.figure(figsize=(12, 10))
sns.heatmap(normalized_conf_matrix, annot=True, fmt=".1f", cmap="Blues",
xticklabels=class_names, yticklabels=class_names)
plt.xlabel("Predicted label")
plt.ylabel("True label")
plt.title("Normalized Confusion Matrix (% by row)")
plt.tight_layout()
plt.show()
# Analyze confusion patterns
print("\nAnalyzing Confusion Patterns:")
def analyze_confusions(normalized_conf_matrix, class_names, threshold=10):
"""Analyze which classes are commonly confused with each other"""
print(f"\nMost Common Confusions (>{threshold}%):")
for i in range(len(class_names)):
for j in range(len(class_names)):
if i != j and normalized_conf_matrix[i][j] > threshold:
print(f"{class_names[i]} confused with {class_names[j]}: {normalized_conf_matrix[i][j]:.1f}%")
# Create confusion matrix without diagonal
confusion_no_diagonal = normalized_conf_matrix.copy()
np.fill_diagonal(confusion_no_diagonal, 0) # Set diagonal to 0 to focus on confusions
# Plot heatmap without diagonal
plt.figure(figsize=(12, 10))
sns.heatmap(confusion_no_diagonal, annot=True, fmt=".1f", cmap="Blues",
xticklabels=class_names, yticklabels=class_names)
plt.xlabel("Predicted label")
plt.ylabel("True label")
plt.title(f"{model_name} - Confusion Matrix (% by row, diagonal removed)")
plt.tight_layout()
plt.show()
# Run confusion analysis
analyze_confusions(normalized_conf_matrix, class_names, threshold=10)
# Print per-class metrics
print("\nPer-class Performance:")
for i, class_name in enumerate(class_names):
true_positives = cumulative_conf_matrix[i, i]
total_actual = np.sum(cumulative_conf_matrix[i, :])
accuracy = true_positives / total_actual if total_actual > 0 else 0
print(f"{class_name}: Accuracy = {accuracy:.4f} (Total samples: {total_actual})")
return {
'mean_accuracy': mean_accuracy,
'std_accuracy': std_accuracy,
'histories': histories,
'accuracies': accuracies,
'confusion_matrix': cumulative_conf_matrix,
'normalized_confusion_matrix': normalized_conf_matrix,
'confusion_no_diagonal': confusion_no_diagonal
}
